{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### for word2vec:\n",
    "https://towardsdatascience.com/implementing-word2vec-in-pytorch-skip-gram-model-e6bae040d2fb\n",
    "### for transD:\n",
    "https://github.com/jimmywangheng/knowledge_representation_pytorch/\n",
    "### for RNN:\n",
    "https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T03:55:03.121808Z",
     "start_time": "2019-07-06T03:55:02.322519Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict, Counter\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import pylab as plt\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from copy import deepcopy\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T03:55:04.269657Z",
     "start_time": "2019-07-06T03:55:04.219692Z"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, input_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "        \n",
    "    def forward(self, input, hidden):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        output, hidden = self.gru(embedded, hidden)\n",
    "        output = self.out(output[0]) \n",
    "        output = self.softmax(output)\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size)\n",
    "    \n",
    "def oneHot(idx):\n",
    "    x = torch.zeros(10).float()\n",
    "    x[idx] = 1.0\n",
    "    return x\n",
    "\n",
    "def tensorsFromPair(pair):\n",
    "    input_tensor = torch.tensor(pair[0], dtype=torch.long).view(-1, 1)\n",
    "    target_tensor = torch.tensor(pair[1], dtype=torch.long).view(-1, 1)\n",
    "    return (input_tensor, target_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T03:55:05.162171Z",
     "start_time": "2019-07-06T03:55:05.092364Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# order is compressed and passed from input sequence to output sequence\n",
    "class EncoderRNN_(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(EncoderRNN_, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, current_input_element_index, order):\n",
    "        # here I call hidden \"order\" because it comressed the order of the input sequences\n",
    "        current_input_element_embedded = self.embedding(current_input_element_index).view(1, 1, -1)\n",
    "        current_input_element_wasted, compressed_order = self.gru(current_input_element_embedded, order)\n",
    "        return current_input_element_wasted, compressed_order\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T06:39:54.198422Z",
     "start_time": "2019-07-06T06:39:54.172570Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderRNN_(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size):\n",
    "        super(DecoderRNN_, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, last_output_element_index, order):# starting for SOS\n",
    "        last_output_element_embedded = self.embedding(last_output_element_index).view(1, 1, -1)\n",
    "        last_output_element_embedded = F.relu(last_output_element_embedded)#only keep positive\n",
    "        current_output_element_embedded, decompressed_order = self.gru(last_output_element_embedded, order)\n",
    "        current_output_element_predicted = self.softmax(self.out(current_output_element_embedded[0]))\n",
    "        return current_output_element_predicted, decompressed_order\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T03:55:06.341207Z",
     "start_time": "2019-07-06T03:55:06.307287Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "input_size, hidden_size, output_size=(10,2,10)# 0 is SOS, both input and output sequences starts from SOS\n",
    "n_iters = 100\n",
    "encoder = EncoderRNN_(input_size, hidden_size)\n",
    "decoder = DecoderRNN_(hidden_size, output_size)\n",
    "learning_rate = 0.05\n",
    "encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "order = encoder.initHidden()\n",
    "criterion = nn.NLLLoss()\n",
    "#\n",
    "input_data=list(zip(range(1,9),range(2,10)))\n",
    "output_data=list(zip(range(10,90,10),range(20,100,10)))\n",
    "pairs= list(zip(input_data,output_data))\n",
    "outputToIndex={v:k for k,v in enumerate([0]+[i for i in range(10,100,10)])}\n",
    "inv_outputToIndex = {v: k for k, v in outputToIndex.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T03:55:07.114511Z",
     "start_time": "2019-07-06T03:55:07.107279Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((1, 2), (10, 20)),\n",
       " ((2, 3), (20, 30)),\n",
       " ((3, 4), (30, 40)),\n",
       " ((4, 5), (40, 50)),\n",
       " ((5, 6), (50, 60)),\n",
       " ((6, 7), (60, 70)),\n",
       " ((7, 8), (70, 80)),\n",
       " ((8, 9), (80, 90))]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T06:31:16.318528Z",
     "start_time": "2019-07-06T03:55:09.276673Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.3804517364501953\n",
      "50 0.22074283361434938\n",
      "100 0.15192389369010925\n",
      "150 0.12909251928329468\n",
      "200 0.10468120813369751\n",
      "250 0.09074642658233642\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-4288ed94fba9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcurrent_output_element_predicted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0mencoder_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mdecoder_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Applications/anaconda/lib/python3.5/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \"\"\"\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Applications/anaconda/lib/python3.5/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Loss=[]\n",
    "for epoch in range(500):\n",
    "    training_pairs = [tensorsFromPair(pair) for pair in pairs]\n",
    "    a=0\n",
    "    for input_tensor,target_tensor in training_pairs:\n",
    "        loss=0\n",
    "        encoder_optimizer.zero_grad()\n",
    "        decoder_optimizer.zero_grad()\n",
    "        input_length = input_tensor.size(0)\n",
    "        target_length = target_tensor.size(0)\n",
    "        \n",
    "        for ei in range(input_length):\n",
    "            current_input_element_wasted, order= encoder(input_tensor[ei], order)\n",
    "            \n",
    "        decoder_input = torch.tensor([[SOS_token]])\n",
    "        \n",
    "        for di in range(target_length):\n",
    "            current_output_element_predicted, order= decoder(decoder_input, order)\n",
    "            topv, topi = current_output_element_predicted.topk(1)\n",
    "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
    "            target=torch.tensor(outputToIndex[target_tensor[di].item()], dtype=torch.long).view(-1, 1)[0]\n",
    "            loss += criterion(current_output_element_predicted, target)\n",
    "\n",
    "        loss.backward(retain_graph=True)\n",
    "        encoder_optimizer.step()\n",
    "        decoder_optimizer.step()\n",
    "        a+=loss.item()\n",
    "        \n",
    "    Loss.append(a/n_iters)\n",
    "    if epoch%50==0:\n",
    "        print(epoch, a/n_iters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T06:31:20.428444Z",
     "start_time": "2019-07-06T06:31:20.225835Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEPCAYAAACDTflkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3X98XXWd5/HXJylpIbe0pLSgLZUqrD6KIy6PbsRVdNCM4Dib4syytow/GK2VB3ZmuiqKyqqrDMoIM+wgDHY7iqKlI7PyIDOgYBnlxyCWgshPcQoyEkBSSG2b0KY0+ewf55zk9Obcc3/k/j7v5+ORR3LPuefme3LbvPP9be6OiIhIJToaXQAREWldChEREamYQkRERCqmEBERkYopREREpGIKERERqZhCREREKqYQERGRiilERESkYrMaXYBaO/LII/3YY49tdDFERFrGvffe+7y7LyzluW0fIsceeyzbtm1rdDFERFqGmf1Hqc9Vc5aIiFRMISIiIhVTiIiISMUUIiIiUjGFiIiIVKztR2dVYsvWETYO7GLH8DgLezpZ0z+Pvt5co4slItJ0FCJ5tmwd4dJNOxnbH+z4ODQ8zqWbdgIoSERE8qg5K8/GgV2TARIZ2+9sHNjVoBKJiDQvhUieHcPjZR0XEckyhUiehT2dicdz3VbnkoiIND+FSJ41/fPoTMiRvfucLVtH6l8gEZEmphDJ09ebo3vO9B/LgXHULyIikkchkmDP6ETi8SH1i4iIHEQhkqBQvwigJi0RkRiFSII1/fMKnrv8up11LImISHNTiCRIm1S4Z9Q547xB1UhERFCIFLQopUlr9+gEl27aqSARkcxTiBSQ1qQFwSx2NW2JSNYpRAro681xeHf6j2fPqOaOiEi2KURSrDtzPrO70meqqzYiIlmmEEnR15vj42cdwdyUJU9UGxGRLFOIFNHXm+OGrx6T2rSl2oiIZJVCpETrzpxf8JxqIyKSVQqREhXraFdtRESySCFSBtVGREQOVvcQMbPTzewxM9tuZucnnF9pZg+Y2f1mts3M3hw796SZPRidq2/Ji9dGvvLtYQWJiGRKXUPEzDqBK4B3AsuB1Wa2PO9ptwInuvvrgQ8CG/POn+rur3f3FTUvcIK02sjEBJrJLiKZUu+aSC+w3d2fcPf9wGZgZfwJ7j7i7tEm592A00SK1Ua0H7uIZEm9Q2Qx8FTs8WB47CBm9m4z+yVwI0FtJOLALWZ2r5mtrWlJUxSbhKh9R0QkK+odIkm/eafVNNz9end/DXAG8KXYqTe5+0kEzWEfNbO3JH4Ts7Vhf8q2HTt2VKPcB4kmIXak/PQuu/aFqn9fEZFmU+8QGQSOiT1eAjxT6MnufjvwKjM7Mnz8TPh5CLieoHks6boN7r7C3VcsXLiwWmU/SF9vjvPf31Pw/MAdo+obEZG2V+8QuQc43syWmVkXsAoYiD/BzI4zMwu/PgnoAl4ws24zmxse7wbeATxU19LnSdt3BLQnu4i0v1n1/GbufsDM1gE3A53AN9z9YTM7Jzx/FfAnwPvN7CVgL/Aed3czOwq4PsyXWcAmd/9hPcufZFFPZ8E+EPWNiEi7s6mBUO1pxYoVvm1b7aaUbNk6wkVXDxc8f3h3B+vOnF+01iIi0izM7N5Sp1FoxvoM9fXm6D+lu+B57YIoIu1MIVIF61cvSD2vXRBFpF0pRKokbU920NpaItKeFCJVsqZ/XtFdEDVaS0TaTV1HZ7WzqOP88ut2smc0ebCCRmuJSLtRTaSKStkFUU1aItJOFCI1kLbSrzrYRaSdKERqIG1OiDrYRaSdKERqJG20lmojItIuFCI1sqZ/XsFzqo2ISLtQiNRIsc2rNNxXRNqBQqSG0jrYh4bHVRsRkZanEKmhYrURraklIq1OIVJjaVvpaj92EWl1CpEai7bSLUSz2EWklSlE6qCvN5c65FdNWiLSqhQidZI25FdNWiLSqhQidZI2i11NWiLSqhQidaQmLRFpNwqROlKTloi0G4VIHalJS0TajUKkztKatC679oU6lkREZOYUInWW1qQ1cMeo+kak5WzZOsKqC57m7ef+hlUXPK1/wxmjEKmztCYtUN+ItJYtW0e4dNNOhobHcYJmWS3nky0KkQZIa9Laob4RaSEbB3Yxtt8POqblfLKl7iFiZqeb2WNmtt3Mzk84v9LMHjCz+81sm5m9udRrW0Vak5Z1aLivtI5Cf/Toj6HsqGuImFkncAXwTmA5sNrMluc97VbgRHd/PfBBYGMZ17aEvt4c/ad0J56bmNDqvtI6FhaoVRc6Lu2n3jWRXmC7uz/h7vuBzcDK+BPcfcTdo/pxN+ClXttK1q9ewGfO7qEj4R1Qc4C0ijX98zhk1sHHZndZam1b2ku9Q2Qx8FTs8WB47CBm9m4z+yVwI0FtpORrW0lfbw6fSD6neSPSCvp6c7z71LmTjxf1dPLxs44oOoBE2ke9QyRpYw2fdsD9end/DXAG8KVyrgUws7Vhf8q2HTt2VFzYekir9qtJS1rBicfNAeCLa49k84WLFSAZU+8QGQSOiT1eAjxT6MnufjvwKjM7spxr3X2Du69w9xULFy6cealrSEuhSKubCFufJxL/pJN2V+8QuQc43syWmVkXsAoYiD/BzI4zMwu/PgnoAl4o5dpWlPZXm0a4SCuYmIg+K0WyaFbxp1SPux8ws3XAzUAn8A13f9jMzgnPXwX8CfB+M3sJ2Au8J+xoT7y2nuWvlUU9nYl9ILnu5G11RZrJZIgoQzKpriEC4O43ATflHbsq9vXFwMWlXtsO1vTP4+JrhhnPy5G9+5wtW0fUxixNLaqBTBQYJCLtTTPWm0Bfb47uOdPfigPjcPl1OxtQIpHSRTUQNWdlk0KkSewZTf4zbs+oa5SWNDU1Z2WbQqRJpA311SgtaWaTo7PUnJVJCpEmkTbUd2h4XLURaVqqiWSbQqRJ9PXmOLy78Nuh9bSkWalPJNsUIk1k3Znzmd2VPKx3bL+rk12a0uToLGVIJilEmkhfb46Pn3VEwfPqZJdmNDXZsLHlkMZQiDSZvt5c6qZV6mSXZjM+2SeiqkgWKUSaUFonu5ZCkWaj0VnZphBpQmmd7FoKRZqNT3asN7Yc0hgKkSa17sz5dCa0au0ZdS679oX6F0ikADVnZZtCpEkVWgoFYOCOUXWwS9PQ2lnZphBpYoWWQgGtqSXNQ5MNs00h0sTSlkLRcF9pFq7JhpmmEGliaaO0QMN9pTmMa7JhpilEmlhfb47+U7oLntdwX2kGExqdlWkKkSa3fvWCgsN9HTjjvEE1a0lDaXvcbFOItIC0NbV2j07w19cMK0ikYSZrIsqQTFKItIBoTa2OAu+WdkCURtIQ32xTiLSIvt4cnvKfVKO1pFHUnJVtsxpdACndwp5OhlI60y+6epivXfc79oxOsLCnkzX98+jrzdWxhJJFWgo+21QTaSFr+uclLoUSt3t0AifYDVEbWUk9aHRWtilEWkhfb45Pva8HK3ENxrH9rrkkUnOasZ5tCpEW09eb49Mf6Cn5+WnNXyLVMDU6SymSRQqRFlRsP/Z8atKSWtLorGybcYiY2XIz+xMze3k1CiSlKbRUfBI1aUktjWt73EwrK0TM7GtmdlXs8R8DvwCuAx4xs/9SwmucbmaPmdl2Mzs/4fyfmtkD4cddZnZi7NyTZvagmd1vZtvKKXu7ifpH5pawSZWWR5FacjVnZVq5NZF3AnfFHv9v4F+AE4GtwOfTLjazTuCK8HWWA6vNbHne034NvNXdXwd8CdiQd/5Ud3+9u68os+xtp683xw1fPSZ1T3YIlkdZdcHTataSmlBzVraVGyJHA08CmNkS4ATgy+7+IPB3QLGaSC+w3d2fcPf9wGZgZfwJ7n6Xu0fTr+8GlpRZxsxZ0z+v4LIoEQ35lVoZ1xDfTCs3RPYC0ey1twK7gahZaQSYW+T6xcBTsceD4bFCPgT8IPbYgVvM7F4zW1voIjNba2bbzGzbjh07ihSp9UXLohSrkWjIr9TChLbHzbRyQ+Q+4KNm9lrgo8CP3CcX41gGPFvk+qQ/lxP/5ZnZqQQh8qnY4Te5+0kEzWEfNbO3JF3r7hvcfYW7r1i4cGGRIrWHvt4cmy9cnPgDjlP/iFSbu5qzsqzcEPkscDJBZ/qrCfosImcQ9IukGQSOiT1eAjyT/yQzex2wEVjp7i9Ex939mfDzEHA9QfOYxKTthgiQK6EjXqQc45psmGllhYi73wMsJfjlvczdH4id3kCRjnXgHuB4M1tmZl3AKmAg/gQzWwp8H3ifu/8qdrzbzOZGXwPvAB4qp/xZUKx/ZO8+LdQo1aUFGLOt7AUY3X0UuDd+zMwWuPuNJVx7wMzWATcDncA33P1hMzsnPH8V8DlgAXClBet7HAhHYh0FXB8emwVscvcfllv+dhctuLhxYFfibPVo2XgtzCjVogUYs62sEDGzDwPz3f2r4ePfI+j4fpmZ/Rz4I3f/bdpruPtNwE15x66Kfb0GWJNw3RMEQ4mliL7eHH29Od5+7m8SO5yiZeMVJFINrtFZmVZun8ifE4zQivwN8DtgPTAP+GKVyiVVkNY/ok2spFrG1ZyVaeWGyFLglwBmNo9gmO8n3f1ygv6Q06pbPJmJNf3zCp7TJlZSLdHQXmVINpUbIp1AVGl9M8Hw3J+Ej58CFlWnWFINxRZqVG1EqmFCa2dlWrkh8u/Au8KvVwF3ufuL4eOXA8PVKphUx7oz5xc8p9qIVIP2E8m2ckPkEmC9mT0PnAVcHjt3KvBA4lXSMMVqI1/59rCCRGZksjlLKZJJ5c4T2UTQD/JlgoUQvx87/RwHh4o0ibTayMQEWlNLZkQ1kWwrez8Rd7/T3S9199vzjn8+HL4rTaZYbWRsv6t/RCqmPpFsKztEzOwwM1tnZteZ2a1m9j0zO9fMDqtFAaU61p05P3Um+55R57JrXyh4XqSQ8cnRWaqKZFG5m1IdTbAI498BK4DDCJZ//xpwr5kdVfUSSlVEK/12pLzjA3eMqllLyqbJhtlWbk3kr4EjgFPcfZm7v9HdlxEM950PXFztAkr19PXmOP/9PanPUbOWlEvNWdlWyc6Gn3b3f4sfdPe7gAuYGv4rTapY/4iG/Uq5olFZruasTCo3RHIkLN0eGmRqwyppYmmjtUC1ESlPtOzJuGoimVRuiDwGvK/AufcSLokiza2vN0f/Kd0Fz+8Zdc44b1A1EinJZJ+IKiKZVO5S8JcA3w470DcR7GR4NMHs9T4KB4w0mfWrF/CT+/ayezT5z8fdoxNcuimokWi1X0kzuRS8UiSTyp1s+B3gHOC1BDsP3gj8A/A64CPhZERpEcWatbQnu5RiXDWRTKtksuEGgnWyTgBOCT8vBp40My170kKKdbIDiRtbicRpdFa2lR0iAO4+4e6Puvu/hZ8nCPYTOaG6xZNaKzYJEdAkREk1NU9EVZEsqihEpH1EkxDndhcOEk1ClDRTQ3wbXBBpCIWI0Neb44avHpP6HA37lUI0xDfbFCIyaVHKdrqahCiFaHRWthUNETN7ZSkfBEN9pYWlbacLqo1Isig71JyVTaXME9lOsA1uMVbi86RJ9fXmeOjxMQbuGE08H9VG2nHeyJatI2wc2MWO4XEW9nSypn9eW95nLUQhMq7//ZlUSoj8Wc1LIU2j2CTEjQO72u6X65atI1y6aSdj+4PfgkPD45poWQYN8c22oiHi7t+qR0Gkeaw7cz4XXT2ceG5oeLztaiMbB3ZNBkgkmmjZTvdZK1qAMdvUsS7TFJuE2G7b6e4oMKGy0HE52IRGZ2Va3UPEzE43s8fMbLuZnZ9w/k/N7IHw4y4zO7HUa6V60iYhttt2ugsLjEordFwONqFNqTKtriFiZp3AFQT7kiwHVpvZ8ryn/Rp4q7u/DvgSsKGMa6VKokmIhbTTkN81/fPoOuTgY7O7rOhoNZlqyuoI/95Qk1b21Lsm0gtsd/cn3H0/sBlYGX+Cu9/l7tGfuXcDS0q9VqqrrzeXOnekXWojfb05Vp92+OTjRT2dfPysI9QfUoKoFtIZ/jNRbSR76h0ii4GnYo8Hw2OFfAj4QbnXmtlaM9tmZtt27Ngxg+JK2l/je0a9bdbV+r1XzQHgv52SY/OFixUgJYpC45BZQVVE/SLZU+8QSWpkT6z/mtmpBCHyqXKvdfcN7r7C3VcsXLiwooJKoFgne7usq7V3LPjtt/8lNceUYyJsvprVaQc9luyod4gMAvFFmpaQsN2umb2OYL+Sle7+QjnXSvUV23fkK98ebvkg2TcW/PLLH+or6aKayKywOctVE8mceofIPcDxZrbMzLoIdkQciD/BzJYC3wfe5+6/KudaqY1itZGJCbjo6uGWbtraG4WIaiJlmQqRsDlLP77MqWuIuPsBYB1wM/Ao8D13f9jMzjGzc8KnfQ5YAFxpZveb2ba0a+tZ/iwrVhuB1m7aUnNWZcbDnvXOqDlLizBmTrl7rM+Yu98E3JR37KrY12uANaVeK/VRbF2tyOXX7WzJTumoOUshUp4oMw6JmrP048sczViXkq1fvYDPnN1DR8q/mlYdsbV3v5qzKhH1gczS6KzMUohIWfp6c5z//p7U5wzcMdpyQRI1ZylEyjOePzpLzVmZoxCRsvX15ug/pTv1Oa3WPzLZnLVff0qXY9roLGVI5ihEpCLrVy9IHbEFrTX0VzWRykwbnaUMzhyFiFSs2IitVhr6u09DfCsSTS6MZqyrNSt76j46S9pHqSO2ovPrVy+oR7FKFt/NcFb4P0Gjs8oT1UQ6Jycb6ueXNQoRmZEoGFotSPJ3M3zpAJOfxyeczo7kZfDlYJpsKGrOkhkrZegvNNeoraTdDCMvHdBvwlJNrZ0VPlafSOYoRKQqShn6C0GQnHHeYMM73NN2LdT6WaXLr4loP5HsUYhI1ZQy9Bdg9+hEwzvc03YtVL9I6SZnrEcd66qJZI5CRKpq/eoFJQUJNLZ5a03/vMLb/ypESjYxuXZW8FhDfLNHISJV1wpBEm3/O/uQqSA5OqydqDmrdJPNWR2asZ5VChGpiXKDpBH9JH29OV5zbNfk46UvCzZaV3NW6aLMiNbOUpdI9ihEpGaiUVtzu4sPl436Sd71P5+qa5iM7J1qf5mfC2siCpGSRTWPQ9SclVkKEampvt4cN3z1mJJrJXvHvK6d7kM7D0x+fecvXgQUIuXIH52l7XGzRyEidVFO8xbUp69ky9YR9oxO/dJ7cV/w9T2P7K3p920n0eTCTm2Pm1masS51U+rs9sjAHaMM3DHK4d0drDtzftU3u9p4w67E41u2vsi6M3vCr6eWRlnY08ma/nktuelWrUw2Z83SjPWsUk1E6qqcfpJIreaV7NiZPOFw92jw53S0NMrQ8DgODA2Pc+mmnQ2fKNlMfLImotFZWaUQkbort58kMnDHaFU73nvmJf/zn3tYcDxpaZSx/c7GgeQaTBZFHenaHje7FCLSMFGtZHZX8edGoo73agwJ7j8luVlqxfLZQOGlUdKWTMmaqOah/USySyEiDdXXm+MHly0tu1ZSjSHBy185B4Aj5nZgwKIjgj+nX3F0kGqFlkZJWzIlayby9lhXc1b2KESkKVTSVwIzGxK858XgN+Alf7mIW69cyua/WsyszqkZ62v653FI3tCT2V3Gmv55ZX+vdjU52VDNWZmlEJGmEfWVlNvEBUF/ydvO/U1ZzVwjYYjkwj6QLVtHGB+Ha2/ZzaoLngbgHW+YqiEt6unk42cdodFZMVMhouasrFKISNOptIkLymvmmgyRQzsmR2JFf0hHI7F2jQT9H92HGpsvXKwAyZM/xFetWdmjeSLStNavXsBrXzWby6/bedCkwFJEzVwXXT3MooT5HVu2jvCdHwajrP7si8+yb78njsTa9ugYAKN7nb1jExw6W393xWl7XKn7/wgzO93MHjOz7WZ2fsL515jZT81szMw+kXfuSTN70MzuN7Nt9Su1NErUxPWvV1ZWM4GgVhGvnUS1jmiG+tDO8cm5Ifn2xYLl+d9pVFa+/OYsZUj21LUmYmadwBXAHwCDwD1mNuDuj8SeNgz8BXBGgZc51d2fr21JpRlFNZNLNw0ztr/866PaiQGV/K67+e4R1qw8ooIrp7TbDPipBRjVJ5JV9W7O6gW2u/sTAGa2GVgJTIaIuw8BQ2b2rjqXTVpAX2+Ovt4cW7aOVNTMBZUFCMC1t+zhX+4cZc/oREUBENWAomazqN8FaNkgyW/O0hDf7Kl3iCwGnoo9HgTeUMb1DtxiZg583d03JD3JzNYCawGWLl1aYVGlmcXDpNKaSb653ZYaSu5TS6KUGgDxmod1TN8+NpoB37IhktecpSG+2VPvEEmaBFDOP7s3ufszZrYI+JGZ/dLdb5/2gkG4bABYsWKF/lm3sWqGyaknHVby4pAQBMDl1+2cDIlct2HYZE3l5BPmcPPPXpyseRRa4TZpBnxSsxfQdE1h0xZgVHNW5tQ7RAaBY2KPlwDPlHqxuz8Tfh4ys+sJmsemhYhkTzxMNg7sYqiCpUnufnhf2dfsGXX2jI5Pfh39TTQ0PF5yIOXPgE9q9rr4mqAv50B4W83SFBbVRDrDITpqzsqeeo/Ougc43syWmVkXsAoYKOVCM+s2s7nR18A7gIdqVlJpSX29OTZfuJh/vXJpWZMWF/V0NmRNrKQZ8EkLP46PTwVIpNGLQW7ZOsI3/zn4/udfMQSoOSuL6hoi7n4AWAfcDDwKfM/dHzazc8zsHAAzO9rMBoGPAReY2aCZHQ4cBdxpZr8AtgI3uvsP61l+aS3RpMViy6nM6gyWOKn3mlhzuixxBnw5YdaoxSCj2lK0dMwLu4LPj/x6rCHlkcap+2RDd78JuCnv2FWxr39L0MyVbzdwYm1LJ+0oauoCpo3qyt/wKt6MBMGoo/Ea/Z5eMK8jsSlqYU9nyc1xScFXj2HESbUlgJ8+pF0hs8a8zeufK1as8G3bNC9RSpP0C/ihx8fK6nAvx2fO7pn2C37L1hEu+e5O9hfZ6312Qk0mvz8FgiDsntNR8dDkJG8/9zcFR8QkrRAgrcXM7nX3FSU9VyEiUtxMOuxLkV8j+vr1O/nHH+0p+PwF8zr4yLvnAweP2No35gVn30eSwqdcqy54OvVnUY3vIY1TTohoISCREkQd9p85u2dyYl01xReOvOzaF7jxzvTFI9960mEA07bvLRYgUJ0O+ZNPmFPz7yGtQQswipQh+su60tnyxewd85Kazr7/4xHMKh8NNZMO+S1bR7j5Zy/W9HtI61CIiJQp3lEPB/ej5LqNl15y9lVhBn0xM2mJznUbqy54uqLO90Kd6vm0A2Q2KEREZig/VKD2fSgz0dkJe/dNTZIsd+JiqTWMPaMTbNk6on6RNqeOdZEayx/xdfIJc7jxrtGaDR2u1Nxu49DZHUVrJ8U61eNmdcIn3zd9BJo0N43OilGISDOq5sKRtTK7yzjtDYdx98P7ZjTkeVFPJ5svXFzDkkq1KURiFCLSzEoNk7l17GtJ09nJQWt4lcKAW6/UatqtpJwQUZ+ISAMlLRzZES4ZnzRp77JrX6jZxMdSFGqC60hY5j6iDvb2ppqISItp1k77tCViNIu9tagmItLGkoYY589bmdMVNDmV0+w0E1FIFGqaa5al66X6FCIiLS5piHEkPjKsq4uadeRHtYy0GlKr7+IoyRQiIm2slFpLtb4PFJ9Dolns7UchIpIhSaEy06HGi2Id50WXsTc0AbHNaAFGkQyLb9wVhUFHGb8V8ndmXNM/j9ldhTcAc2dyocktW9MXmZTWoNFZIpIof00ww9g9OpE6BDl+XTmjx/KXwpfG0mTDGIWISGOkbVxViAGOhgQ3mvYTEZGGq2SSYRQ6Q8PjXHT1MG879zeccd6gmr6amGoiIlITW7aOcPE1w1VfaHKmTV+FVgc4+YQ53P3wvqKrBmSBmrNiFCIijVOvhSZLDZakPeiLyeJWvwqRGIWISOPVan5KkrRAOeO8wZK2EM6XtZWIFSIxChGR5lLPQIkc3t3B7590aMWLV0YrEefvDdOuTV0KkRiFiEjzakSgVGJqbbDpTWH9p3SzfvUCYPoGZK0aMgqRGIWISOtoxlCJ+kTS5r585uwegMSQacU5ME0dImZ2OvB/gE5go7t/Je/8a4BvAicBn3X3S0q9NolCRKR1NUuozOlixhuCzSRMkrZYzt9xspoh1bQhYmadwK+APwAGgXuA1e7+SOw5i4BXAGcAO6MQKeXaJAoRkfbQLIEyU/2ndPPaV80uudmrlBFl1R5B1swh8kbgC+5+Wvj40wDu/uWE534BGImFSMnXxilERNpXuwQLwEmv7mJwx/i0YFl1wdMlLSFj4XT/atRMmnlTqsXAU7HHg8Ab6nCtiLShei11Xw/3PTbVXhbN2H/o8bGS1yCL6gPRtRddPVyXyZL1DpGk5T1LfbdLvtbM1gJrAZYuXVriy4tIq0vaoCspWKIhvzf/7MVpzUSHzjb2jjVHCFU6JDlSjx0l6x0ig8AxscdLgGeqfa27bwA2QNCcVX4xRaRdpO38mNQ3AdRkuZZGqfWOkvUOkXuA481sGfA0sAo4qw7XiohMkxYw9ViupV5quaNkXUPE3Q+Y2TrgZoJhut9w94fN7Jzw/FVmdjSwDTgcmDCz9cByd9+ddG09yy8i2RCFS6E9VfKZTfVJNKNKVlQulSYbioiUKWlmOnBQ4Ozd5xxogiaxWZ3wyff1lNWc1cyjs0REWl6hZrD8kWKFZrlH8zog2C64lg6dY201OktEJBPiQZO2ptZDj4+VPAqrkmazkRoPd1aIiIjUWFoH/vrVCw4aJZbrNl56yactszK7yzjtDYclDktOU8v+EFCIiIg0XKH5LUm1l/zAMYw9oxOJ/TCzu2yyv6ZW1LEuItImqrUUvTrWRUQyKK3ZrFY66vrdRESkrShERESkYgoRERGpmEJEREQqphAREZGKtf0QXzPbAfxHhZcfCTxfxeI0u6zdL+iesyBr9wszv+dXuPvCUp7Y9iEyE2a2rdSx0u0ga/cLuucsyNr9Qn3vWc1ZIiJSMYWIiIhUTCGSbkOjC1BnWbtf0D1nQdbuF+p4z+oTERGRiqkmIiIiFVOIJDCz083sMTPbbmaS54LOAAAHyklEQVTnN7o8tWJmT5rZg2Z2v5ltC4/1mNmPzOzfw89HNLqcM2Fm3zCzITN7KHas4D2a2afD9/0xMzutMaWuXIH7/YKZPR2+z/eb2R/GzrX0/QKY2TFm9mMze9TMHjazvwyPt+X7nHK/jXmf3V0fsQ+gE3gceCXQBfwCWN7octXoXp8Ejsw79tfA+eHX5wMXN7qcM7zHtwAnAQ8Vu0dgefh+zwaWhf8OOht9D1W43y8An0h4bsvfb3gfLwNOCr+eC/wqvLe2fJ9T7rch77NqItP1Atvd/Ql33w9sBlY2uEz1tBL4Vvj1t4AzGliWGXP324H8TawL3eNKYLO7j7n7r4HtBP8eWkaB+y2k5e8XwN2fdff7wq/3AI8Ci2nT9znlfgup6f0qRKZbDDwVezxI+hvUyhy4xczuNbO14bGj3P1ZCP6xAosaVrraKXSP7fzerzOzB8LmrqhZp+3u18yOBf4z8DMy8D7n3S804H1WiExnCcfadQjbm9z9JOCdwEfN7C2NLlCDtet7//fAq4DXA88Cl4bH2+p+zSwH/D9gvbvvTntqwrGWu++E+23I+6wQmW4QOCb2eAnwTIPKUlPu/kz4eQi4nqCK+5yZvQwg/DzUuBLWTKF7bMv33t2fc/dxd58A/i9TTRltc79mdgjBL9Tvuvv3w8Nt+z4n3W+j3meFyHT3AMeb2TIz6wJWAQMNLlPVmVm3mc2NvgbeATxEcK8fCJ/2AeCGxpSwpgrd4wCwysxmm9ky4HhgawPKV1XRL9LQuwneZ2iT+zUzA/4BeNTd/yZ2qi3f50L327D3udEjDZrxA/hDghEPjwOfbXR5anSPryQYsfEL4OHoPoEFwK3Av4efexpd1hne57UEVfuXCP4i+1DaPQKfDd/3x4B3Nrr8Vbrfa4AHgQfCXygva5f7De/hzQTNMw8A94cff9iu73PK/TbkfdaMdRERqZias0REpGIKERERqZhCREREKqYQERGRiilERESkYgoRaXlmdraZuZn9Ln/VYTObFZ77QhW/3wfDlWH3m9nvqvW69WJmx4Y/kzWNLou0PoWItJN5wKdq+Q3M7OUEu8bdBbwN6Kvl9xNpdgoRaSe3AH9uZkfX8HscT7BdwLfc/U5331bD7yXS9BQi0k4uDD9/tpKLzezVZnZ92Cy218zuNrPTY+evBn4SPrw1bBK6ushr/nH4Oi+Gr3udmS3Ne86TZvYdM/twuHHQPjO7z8xOTXi995rZL8LnPG9m1+QtdxE978Pha+w1s51mdpuZ/de8p3Wa2RfN7NmwbP9sZkvyXucsM/u5mY2Y2S4LNjH7SNo9S7YoRKSdPAt8DVhrZq8o58KwmepO4ERgHfA/gN8BN5rZO8OnfQn4i/DrjwJvDI8Ves1zCBbJewT478BHgNcCt0XrlsW8FfgYQQCuAsaAH5jZq2Ovt5ZgaYtHgT8m2GjptPD1crHnXULQ5HZfeB/vBW4HDgov4NPAccAHgb8M7+e7sdd5M/Ad4DaCvTjOJFjYb36he5YMavQ6MPrQx0w/gLMJ1hI6Dugh+OX/jfDcrPDcF4q8xiXAAeC42LFOgrWG7osd6wtf7/eLvF4O2BWVI3b8WGA/wfLd0bEnw2NLY8fmEmwudU2sLM8BP857vWgdpb8IHx8HjAN/k1K2Y8Nrbss7/onw+Mtjj4cb/f7qo7k/VBORtuLuwwT7KLw//ld8Cd4C3O3u22OvNU6woOHrzezwMovyRuBw4LvhCLFZZjaLYFHEX4bfL+5ud/9N7HvvAW4MXwfg1QSbKn03fpG73wn8B0FNBoKQ6yCoiRRzY97jB8PPUY3lHuCIsKntj8xMNRCZRiEi7ehvCf6K/2IZ1/QQNIfl+y3Bpj5HJJxLE+2it4VgRd34x+8RrDAb91zCazzH1A50PeHnQmWMzkevO1hCGfO30R0LP88BcPfbCJqwjiHYb2aHmW0xs9eV8NqSEbMaXQCRanP3ETP7MkGN5KslXjYMJI3qOpqgiafUfcsjL4SfzyZYaj/fnrzHRyU85yjg6Vj5ovLkOxqIRok9H35eTNAUNyPu/k/AP4V9Lr8PXAz80MyWeLD5kWScaiLSrq4k+AV8YbEnhm4DTg73rAbAzDqB9wA/D5uXynEXQVAc5+7bEj7yf8GfbGaTu8+FHe/vAn4aHnqMoGayKn5ROOLqFWH5Iaj5TABryyxvKncfcfd/Ab4OvIzpNSnJKNVEpC25+5iZfZHS+gYgaAI7G/iRmX0e2A2cC/wngl/m5X7/3WZ2HnCFmS0EfkDQ0b6YoP/iJ+6+KXbJc8At4cz6MYJJk92Eo7/cfdzMPgd83cy+QzBqajHwVwSbLn0zfN7jZva3wMfCIBog6GjvBX7p7v9Y6j2EP7+jgB8TbKe6hGB02v3uvqPcn4m0J4WItLNvAucRTBBM5e7PhENaLwb+HphNsGPcu9z9h5V8c3f/upk9FZbhLOAQgtrR7eFrx91GMAflIoJf1o8Q7ED3q9jrbTCzF8PXuwEYAW4CPunuI7HnfcLMthOE4AeAUYLd7m4p8xZ+RhAaf0vQ5zIUvsb/KvN1pI1pZ0ORBjOzJ4E73f29jS6LSLnUJyIiIhVTiIiISMXUnCUiIhVTTURERCqmEBERkYopREREpGIKERERqZhCREREKqYQERGRiv1/WnaukG1DL1IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Loss,color='RoyalBlue',marker='o')\n",
    "plt.xlabel('N of epochs',fontsize=16)\n",
    "plt.ylabel('Loss',fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T06:31:24.075510Z",
     "start_time": "2019-07-06T06:31:24.049400Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2) (10, 20) predicted: [20, 20]\n",
      "(2, 3) (20, 30) predicted: [20, 30]\n",
      "(3, 4) (30, 40) predicted: [30, 40]\n",
      "(4, 5) (40, 50) predicted: [40, 50]\n",
      "(5, 6) (50, 60) predicted: [50, 60]\n",
      "(6, 7) (60, 70) predicted: [60, 70]\n",
      "(7, 8) (70, 80) predicted: [70, 80]\n",
      "(8, 9) (80, 90) predicted: [80, 80]\n"
     ]
    }
   ],
   "source": [
    "for k,v in pairs:\n",
    "    input_tensor,target_tensor=tensorsFromPair((k,v))\n",
    "    for ei in range(input_length):\n",
    "        current_input_element_wasted, order= encoder(input_tensor[ei], order)\n",
    "    decoder_input = torch.tensor([[SOS_token]])\n",
    "    predictates=[]\n",
    "    for di in range(2):\n",
    "        current_output_element_predicted, order= decoder(decoder_input, order)\n",
    "        topv, topi = current_output_element_predicted.topk(1)\n",
    "        predictates.append(inv_outputToIndex[topi.item()])\n",
    "        decoder_input = topi.squeeze().detach()  # detach from history as input\n",
    "    print(k,v,'predicted: '+str(predictates))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T06:37:52.916606Z",
     "start_time": "2019-07-06T06:37:52.686877Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPIAAAD8CAYAAABNYvnUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAFldJREFUeJzt3XtwVeW9xvHvTgJogKRlCBoI7EAjsMnFSKJcSq0IgeEmAi2NZ+uAtxyO4Ch0PHWaacFqABn0gAJ6QrFyiXgpKigXy01aEItRaJITpTCSkAQo0BYkIhiTff5YEyAXJMlaO2vvtZ/PzJ5kLXbe9zfMPHnftfab9bp8Pp8PEQlqYXYXICLmKcgiDqAgiziAgiziAAqyiAMoyCIOoCCLOICCLOIACrKIA0TY0Wnnzp2Jj4+3o2uRoFJSUsLp06ev+T5bghwfH09+fr4dXbee6mqYMwfWrIHjxyE2Frxe41yELf/tEoTS09Ob9D5Nrf3l2Wdh6VJ44QX44gtYvNg4njfP/32fOwePPw5uN1x/PQweDJ984v9+xTYaGvzlo49g3DjjBRAfD3fdBX/9q//7fughKCiAlSshLs6YFQwfDsXF0K2b//uXVmd6RL5w4QK33XYbN998M4mJicyePduKuoLfkCGwc6cxGoMRoh07YPRo//b7zTewbh3Mnw933AEJCcZ0PiEBXnrJv32LbUyPyO3atWPHjh106NCBqqoqhgwZwqhRoxg4cKAV9QWvX/3KmOL26wfh4fDdd5CdDY884t9+v/vOuD6/7rq656+/Hnbv9m/fYhvTI7LL5aJDhw4AVFVVUVVVhcvlMl1Y0HvjDVi1Cl57DT77zPh+2TJYscK//XbsCIMGwTPPQEWFEeo1a2DvXuOmmziSJTe7qqurSU1NpUuXLmRkZDBgwIAG78nNzSU9PZ309HROnTplRbcBJ68wj/hF8YQ9Fcax/7qX/P+4AzIzITkZ7rsPZs1qnZtdq1dDWJhxfdyunXHD7Z57jJmBOJIlQQ4PD+fAgQOUl5ezb98+ioqKGrwnKyuL/Px88vPziYmJsaLbgJJXmEfWe1mUni3Fh492F6t5rfgN8grzLr8pPBxqavxfzI9+BLt2QWUllJXBvn1QVQU9e/q/b7GFpR8//eAHP+COO+5gy5YtVjYbFLK3Z3O+6vyl4/d6w6xdVWx7YSaUlMA778Dzz8OECZb3feVMIH5R/OVfHu3bG59f//vf8MEHMH685X1LYDAd5FOnTnHmzBkAvvnmG7Zt20bfvn1NFxZsjp49Wuf40dHwx34w561T4PHAL38JDz8MOTmW9lt/JlB6tpQ3n3+Qncv+G44cga1bYehQ6NMH7r/f0r4lcJi+a338+HGmTJlCdXU1NTU1TJ48mbFjx1pRW1DpEd2D0rOll44r28HMUbAo003J4yV+67f+TACgXeVFev32f2DmYujUCSZNMn6BtGnjtzrEXqaDnJKSwv79+62oJajlDMsh672sOqGKbBNJzjBrR+D66s8EAN5Kgj8mVVMzu8qvfQeEP/8ZFi6ETz+FY8fgD3+AqVPtrqrVaYmmRbzJXnLH5eKOduPChTvaTe64XLzJXr/22yO6R7POO05lJSQlGUtgr7/e7mpsoyWaFvIme/0e3PrsmgkEjNGjL6+WC8GRuJZG5CBn10xAAotGZAewYyYggUUjsogDKMgSdK66ACaEaWotQaV2AUztzb3Ss6VkvZcFQChfXGhElqBSfwFM+4tw09Hz5K16wljHfvQoHDhgfA0hCrIElfoLYNKPwYH/hU0LjxsPVZg9G265BX77W5sqtIem1hJU6i+F3dUTXHPAHe3fpbCBTiOyBJWcYTlEtomscy6kFsBcRWgFOT4eXK6GrzFj7K5MmkgLYBoXWlPrTz4xHn1T6/hxSEuDyZPtq0maTQtgGgqtINd/MsmKFRAVBT//uT31iFgktKbWV/L5jCDfey9ERl77/SIBLHSDvHWr8QSNhx6yuxIR00I3yMuXw623Qmqq3ZWImBaaQT55EtavN56hJeIApoNcVlbG0KFD8Xg8JCYmsnjxYivqskyjC+xffdV43nNmpt3liVjC9F3riIgInnvuOfr378+5c+dIS0sjIyODfv36WVGfKY0usN/wMOOWRxOVmWnsyiDiAKZH5NjYWPr37w9Ax44d8Xg8VFRUmC7MCo09YfK2Q98QVXpC02pxFEs/Ry4pKWH//v1X3TImNzcXoNW2jGnsCZMf9oSwOS5qbrutVWoQaQ2W3eyqrKxk0qRJLFq0iKioqAb/bseWMSH/hEkJGZYEuaqqikmTJuH1epk4caIVTVpCC+wlVJgOss/n48EHH8Tj8TBr1iwrarKMFthLqHD5fD6fmQZ2797NT37yE5KTkwkLM34vzJ07l9G1zxpuRHp6Ovn5+Wa6FQkJTc2K6ZtdQ4YMweTvAhExKTRXdok4jIIs4gAKsogDKMgiDqAgiziAgiziAAqyiAMoyCIOoCCLOICCLOIACrKIAyjIIg6gIIs4gIIs4gAKsogDKMgiDqAgiziAgiziAJYE+YEHHqBLly4kJSVZ0ZyINJMlQZ46dSpbtmyxoikRaQFLgnz77bfTqVMnK5oSkRbQNbKIP8ybZ+y/HRUFMTEwbhwUFfmtu1YLcm5uLunp6aSnp7fa3k8itvnwQ3jkEfjoI9ixAyIiYPhw+Ne//NKdpZu4fZ+srCyysrIA46HbIo72wQd1j1evhuho2LPHGJ0tpqm1SGs4dw5qauCHP/RL85YE+Z577mHQoEEcPHiQuLg4VqxYYUWzIs7x2GOQmgqDBvmleUum1mvXrrWiGRFnmjULdu82XuHhfulCU2sRi+QV5hG/KJ6wp8KIXxRPXmEezJwJa9caN7x69fJb3612s0vEyfIK88h6L4vzVecBKD1bytn/nMo3ByO5fvfH0LevX/vXiCxigezt2ZdCDLBkI9z76Xc8kHmdcYPrxAnjVVnpl/4VZGm6ZcugZ0+47jpIS4O//MXuigLG0bNH6xxP/wSivoW1y05CbOzl18KFfulfU2tpmjfeMO68LlsGQ4YYX0eNguJi6NHD7ups1yO6B6VnSy8du+YYX93RbkoeL/F7/xqRpWmefx6mToWHHwaPB1580RhhXnrJ7soCQs6wHCLbRNY5F9kmkpxhOa3Sv4Is1/btt/DppzBiRN3zI0YYSxAFb7KX3HG5uKPduHDhjnaTOy4Xb7K3VfrX1Fqu7fRpqK6GG26oe/6GG2DbNntqCkDeZG+rBbc+jcjSdC5X3WOfr+E5sYWCLI26cnHDTa8NoiY8zPj45EonTzYcpcUWCrI0ULu4ofRsKT58HP76KJ/GwqE36t3Y2roVBg+2p0ipQ0GWBuovbgBYOLAG97s74fe/h88/Nz6KOnYMpk2zqUq5km52SQP1FzcAvJkEnc/D0meegePHISkJNm0Ct9uGCqU+BVkaqL+4odbGDDdLN5a0fkFyTZpaSwN2L26Q5lOQpQG7FzdI82lqLY2yc3GDNJ9GZBEHsCTIW7ZsoU+fPiQkJDB//nwrmhSRZjAd5OrqaqZPn87mzZspLi5m7dq1FBcXW1GbiDSR6SDv27ePhIQEevXqRdu2bcnMzGT9+vVW1CYiTWQ6yBUVFXTv3v3ScVxcHBUVFWabFZFmMH3X2ufzNTjnauQvYnJzc8nNzQXQljEiFjM9IsfFxVFWVnbpuLy8nK5duzZ4X1ZWFvn5+eTn5xMTE2O2WxG5gukg33rrrRw6dIgjR47w7bff8vrrr3PXXXdZUZuINJHpqXVERARLlixh5MiRVFdX88ADD5CYmGhFbSLSRJas7Bo9ejSjR4+2oikRaQGt7BJxAAVZxAEUZBEHUJBFHEBBFnEABVnEARRkEQdQkEUcQEEWcQAFWcQBFGQRB1CQRRxAQRZxAAVZxAEUZBEHUJBFHEBBFnEABVnEAUwF+a233iIxMZGwsDDy8/Otqqmu48dhyhSIiYHrroN+/WDXLv/0JRKkTAU5KSmJt99+m9tvv92qeuo6cwZ+/GPw+WDjRvj8c3jxRejSxT/9iQQpUw/f83g8VtXRuAULIDYWVq26fK5nT//2KRKEAvsa+d13YcAA+MUvjFE4NRWWLDFGaBG55Joj8vDhwzlx4kSD8zk5OYwfP77JHbVoy5gvv4Rly2DmTHjySThwAB591Pi3GTOa3LeI010zyNu2bbOko6ysLLKysgBIT09v2g/V1EB6OsybZxzfcgscOgRLlyrIIlcI7Kl1bKxxl/pKHg8cPWpPPSIBylSQ33nnHeLi4ti7dy9jxoxh5MiRpgvKK8wjflE8YU+Fsb7zaU5+9pe6b/j738HtNt2PiJOYCvKECRMoLy/n4sWL/OMf/+CDDz4wVUxeYR5Z72VRerYUHz6eSTvPD//2dw48OhkOH4a33oIXXoDp0031I+I0ATW1zt6ezfmq85eO87vB3ZnQ7u13ISkJsrPh6afhkUdsrFIk8FiyiZtVjp5teO27qTds7v0dNbNrbKhIJDgE1IjcI7pHs86LiCGggpwzLIfINpF1zkW2iSRnWI5NFYkEh4AKsjfZS+64XNzRbly4cEe7yR2XizfZa3dpIgEtoK6RwQizgivSPAE1IotIyyjIIg6gIIs4gIIs4gAKsogDKMgiDqAgiziAgiziAAqyiAMoyCIOoCCLOICCLOIApoL8xBNP0LdvX1JSUpgwYQJnzpyxqi4RaQZTQc7IyKCoqIiCggJ69+7NvNrH1opIqzIV5BEjRhARYfwl5MCBAykvL7ekKBFpHsuukV955RVGjRplVXMi0gyWbBmTk5NDREQEXu/VHwjQoi1jRKRJXD6fuR3RVq5cycsvv8z27duJjIy89g9gbBnjt/2URRykqVkx9aifLVu28Oyzz7Jr164mh1hErGfqGnnGjBmcO3eOjIwMUlNTmTZtmlV1iUgzmBqRDx8+bFUdImKCVnaJOICCLOIACrKIAyjIIg6gIIs4gIIs4gAKsogDKMgiDqAgiziAgiziAAqyiAMoyCIOoCCLOICCLOIACrKIAyjIIg6gIIvYaelSSEmBqCjjNWgQbNzY7GYUZBE7xcXBs8/CZ59Bfj7ceSfcfTcUFDSrGVNB/s1vfkNKSgqpqamMGDGCY8eOmWlOJPSMHw+jRkFCAvTuDTk50LEj7N3brGZM7/1UUFDAgQMHGDt2LL/73e/MNCcS2qqr4fXXobISBg9u1o+aevheVFTUpe+//vprXC6XmeZEQlNhoXFtfOECdOgA77wDycnNasJUkAGys7NZtWoV0dHR7Ny502xzIqGnTx84cADOnIF162DKFPjwQ0hKanIT19xpoilbxgDMmzePCxcu8NRTTzXaTv0tY0pLS5tcpEhIGT4c3G5YsaLJO02Y3jKmVmlpKWPGjKGoqOia79WWMRLK8grzyN6ezdGzR+kR3YOcYTl4k6/YN+3OO6FrV1izpnW2jDl06BA33XQTABs2bKBv375mmhNxvLzCPLLey+J81XkApv2xlFeLH6T9xFPc3W0YvPaaMa1u5mfJpoL85JNPcvDgQcLCwnC73bz88stmmhNxvOzt2ZdCDHBjJfz+zYvc+MpM6BRjLA7ZvBlGjmxWu6aCvG7dOjM/LhJyjp49Wuf4/gnGVxcuamafbHG7Wtkl0op6RPdo1vmmUpBFWlHOsBwi29TdgjiyTSQ5w3JMtasgi7Qib7KX3HG5uKPduHDhjnaTOy637l3rFjC9IEREmseb7DUd3Po0Ios4gIIs4gAKsogDKMgiDqAgiziAgiziAAqyiAMoyCIOoCCLOICCLOIACrKIAyjIIg6gIIs4gIIs4gCWBHnhwoW4XC5Onz5tRXMi0kymg1xWVsbWrVvp0cPco0pEpOVMB3nmzJksWLBA28WI2MhUkDds2EC3bt24+eabrapHnGDuXHC5YMYMuysJGdd81M/3bRkzd+5c/vSnPzWpo/pbxohDffwxLF9uPJ9ZWo+vhQoKCnwxMTE+t9vtc7vdvvDwcF/37t19x48fv+bPpqWltbRb/5k92+eDuq8bbrC7quBy5ozP16uXz7d9u8/305/6fNOn211R0GtqVlr88L3k5GROnrz8QO34+Hjy8/Pp3LmzJb9gbNGnj7FdR63wcNtKCUpZWfCznxl7F2mv7Falp2heKSICbrzR7iqC0/LlcPgwrF5tdyUhybIFISUlJcE9GgN8+SV06wY9e0JmpnEs13bwIPz615CXB23b2l1NSNLKrloDBsCrrxobaC1fDidOwODB8M9/2l1Z4Nu7F06fNjbmjogwXrt2wbJlxvcXL9pdoeNpal1r1Ki6xwMHQq9esHIlzJplT00B7Mo9fpPaxjF33TzG9h57+Q333w833WSM1Bql/U4j8tV06ACJiXDokN2VBJzaPX5Lz5biw0fht2X84ounyfP9zRiVk5KgfXvo1Mn4XouF/C6kg5xXmEf8onjCngojflE8eYV5l//xwgX44guIjbWvwABVf49fgPNV58nenm1TRRKyU+v6O8fPeLOUVf/3IO0nnuTuToPh6afh669hyhSbKw089ff4bfT8lR/jid+F7Ihcf1SJ+wr+8PpFRo+bBRMnQrt2xiolt9vGKgOTv/b4lZYL2RG5/qhyz8+Nr8bO8RU2VBQ8cobl1JnNgDV7/ErLheyIrFGl5fy1x6+0XMiOyBpVzPHHHr/SciE7ImtUEScJ2REZNKqIc4TsiCziJAqyiAMoyCIOoCCLOICCLOIALp/P52vtTjt37kz79u2JiYlp7a5b7NSpU6rXj1Rv40pKSpq08YMtQQZIT08nPz/fjq5bRPX6l+o1R1NrEQdQkEUcIHzOnDlz7Oo8LS3Nrq5bRPX6l+ptOduukUXEOppaiziA7UEOlr2Vn3jiCfr27UtKSgoTJkzgzJkzdpfUqC1bttCnTx8SEhKYP3++3eV8r7KyMoYOHYrH4yExMZHFixfbXVKTVFdXc8sttzB27Nhrv7mV2BrkYNpbOSMjg6KiIgoKCujduzfz5s2zu6QGqqurmT59Ops3b6a4uJi1a9dSXFxsd1lXFRERwXPPPcfnn3/Oxx9/zNKlSwO63lqLFy/G4/HYXUYdtgY5mPZWHjFiBBERxl99Dhw4kPLycpsramjfvn0kJCTQq1cv2rZtS2ZmJuvXr7e7rKuKjY2lf//+AHTs2BGPx0NFRWA/Zqm8vJyNGzfy0EMP2V1KHbYFOZj3Vn7llVcYVf+B9gGgoqKC7t27XzqOi4sL+GDUKikpYf/+/QwYMMDuUr7X448/zoIFCwgLs/2qtA6/PljAqr2VW8v31Tt+/PhL30dEROD1Bt4DCRr7ACIYZjuVlZVMmjSJRYsWERUVZXc5V/X+++/TpUsX0tLS+DDAHvfr1yBv27at0fOFhYUcOXLk0mhcXl5O//792bdvHzfauBvi1eqttXLlSt5//322b98ekAGJi4ujrKzs0nF5eTldu3a1saJrq6qqYtKkSXi9XiZOnGh3Od9rz549bNiwgU2bNnHhwgW++uor7r33XtasWWN3aS3f6NxKbrfbd+rUKbvL+F6bN2/2eTwe38mTJ+0u5aqqqqp8PXv29H355Ze+ixcv+lJSUnxFRUV2l3VVNTU1vvvuu8/32GOP2V1Ks+3cudM3ZswYu8u4JLAm+gFsxowZnDt3joyMDFJTU5k2bZrdJTUQERHBkiVLGDlyJB6Ph8mTJ5OYmGh3WVe1Z88eVq9ezY4dO0hNTSU1NZVNmzbZXVZQ0souEQfQiCziAAqyiAMoyCIOoCCLOICCLOIACrKIAyjIIg6gIIs4wP8Dh0jX1cYnCv8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pos=dict(zip(range(10),decoder.out.weight.data.numpy()))\n",
    "fig = plt.figure(figsize=(8, 4),facecolor='white')\n",
    "ax = fig.add_subplot(121)\n",
    "for i in pos:\n",
    "    x,y=pos[i]\n",
    "    plt.plot(x,y,marker='o',color='g')\n",
    "    plt.text(x,y,i,size=14,color='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
